package com.shujia.spark.core

import com.shujia.spark.util.HDFSUtil
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable.ListBuffer

object Demo4MakeWords {
  def main(args: Array[String]): Unit = {
    //创建环境
    val conf = new SparkConf()
    conf.setMaster("local")
    conf.setAppName("cache")

    val sc = new SparkContext(conf)

    val linesRDD: RDD[String] = sc.textFile("data/students.txt")

    val resultRDD: RDD[String] = linesRDD.flatMap(line => {

      val listBuffer = new ListBuffer[String]

      var i = 0
      while (i <= 10000) {

        listBuffer += line

        i += 1
      }

      listBuffer
    })

    HDFSUtil.deletePath("data/students")

    resultRDD.saveAsTextFile("data/students")

  }

}
